# _*_ coding:utf-8 _*_
# @File  : futures_daily_quotes.py
# @Time  : 2021-06-25  09:03
# @Author: zizle
import json
import os
import datetime
import requests
import pandas as pd
import numpy as np
from hutool import utils as hu_tool
from logger import logger


class CZCEDailyQuotes(object):
    def __init__(self, parse_date: datetime.datetime):
        self.temp_folder = 'temp/czce/daily'
        self.parse_year = parse_date.strftime("%Y")
        self.parse_day = parse_date.strftime("%Y%m%d")
        self.xls_url = f'http://www.czce.com.cn/cn/DFSStaticFiles/Future/{self.parse_year}/{self.parse_day}/FutureDataDaily.xls'
        self.headers = {
            'User-Agent': 'User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
            'Host': 'www.czce.com.cn',
        }
        self.filepath = f'{self.temp_folder}/{self.parse_day}.xls'
        if not os.path.exists(self.temp_folder):
            os.makedirs(self.temp_folder)

    def get_quotes_xls_file(self):
        try:
            r = requests.get(url=self.xls_url, headers=self.headers)
            content = r.content
            if r.status_code == 200:
                with open(self.filepath, 'wb') as f:
                    f.write(content)
            else:
                if os.path.exists(self.filepath):
                    os.remove(self.filepath)
        except Exception as e:
            logger.error(f'Spider {self.parse_day} CZCE daily-quotes Error:{e}')
        else:
            logger.info(f'Spider {self.parse_day} CZCE daily-quotes Successfully!')

    def get_daily_quotes(self):
        if not os.path.exists(self.filepath):
            self.get_quotes_xls_file()
        # 执行分析
        xls_df = pd.read_excel(self.filepath, thousands=',', skiprows=[0])
        # xls_df = xls_df[~xls_df['品种月份'].str.contains('总计|小计')]  # 选取品种月份不含有小计和总计的行
        xls_df = xls_df[~xls_df['合约代码'].str.contains('总计|小计')]  # 合约代码  2021.04.14变化
        xls_df['品种代码'] = xls_df['合约代码'].apply(lambda x: x[:2])  # 变为品种
        # 2020-09-07 空盘量->持仓量
        if xls_df.columns.tolist() != [
            '合约代码', '昨结算', '今开盘', '最高价', '最低价', '今收盘', '今结算', '涨跌1', '涨跌2', '成交量(手)', '持仓量',
            '增减量', '成交额(万元)', '交割结算价', '品种代码'
        ]:
            logger.error('Parser CZCE Daily-Quotes Error: columns error!')
            return [], 'Parser-Error: columns error!'
        xls_df = xls_df.fillna(0)
        xls_df.columns = [
            "contract", "pre_settlement", "open_price", "highest", "lowest", "close_price", "settlement", "zd_1",
            "zd_2", "trade_volume", "position_volume", "increase_volume", "trade_price", "delivery_price", "variety_en"
        ]
        # 将品种月份处理为4位合约(分离字母数字后插入年份的第3个数)
        xls_df["contract"] = xls_df["contract"].apply(hu_tool.modify_contract_express, args=(self.parse_day,))
        del xls_df['zd_1']
        del xls_df['zd_2']
        del xls_df['delivery_price']
        daily_data = xls_df.to_dict(orient='records')
        return daily_data, 'Parser Successfully!'


class SHFEDailyQuotes(object):
    def __init__(self, parse_date: datetime.datetime):
        self.temp_folder = 'temp/shfe/daily'
        self.parse_year = parse_date.strftime("%Y")
        self.parse_day = parse_date.strftime("%Y%m%d")
        self.url = f"http://www.shfe.com.cn/data/dailydata/kx/kx{self.parse_day}.dat"
        self.headers = {
            'User-Agent': 'User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
            'Host': 'www.shfe.com.cn',
        }
        self.filepath = f'{self.temp_folder}/{self.parse_day}.json'
        if not os.path.exists(self.temp_folder):
            os.makedirs(self.temp_folder)

    def get_quotes_json_file(self):
        try:
            r = requests.get(url=self.url, headers=self.headers)
            content = r.json()
            if r.status_code == 200:
                with open(self.filepath, 'w', encoding='utf8') as f:
                    json.dump(content, f)
            else:
                if os.path.exists(self.filepath):
                    os.remove(self.filepath)
        except Exception as e:
            logger.error(f'Spider {self.parse_day} SHFE daily-quotes Error:{e}')
        else:
            logger.info(f'Spider {self.parse_day} SHFE daily-quotes Successfully!')

    def get_daily_quotes(self):
        if not os.path.exists(self.filepath):
            self.get_quotes_json_file()
        with open(self.filepath, 'r', encoding='utf8') as f:
            source_content = json.load(f)
        # 解析content转为DataFrame
        json_df = pd.DataFrame(source_content['o_curinstrument'])
        # 选取PRODUCTID非总计、DELIVERYMONTH非小计的行
        json_df = json_df[~json_df['PRODUCTID'].str.contains('总计|小计|合计|_tas|efp')]  # 选取品种不含有小计和总计合计_tas的行
        json_df = json_df[~json_df['DELIVERYMONTH'].str.contains('总计|小计|合计')]  # 选取合约不含有小计和总计合计的行
        # 处理空格
        json_df["PRODUCTID"] = json_df["PRODUCTID"].apply(lambda x: x.split("_")[0].upper())
        # json_df["PRODUCTGROUPID"] = json_df["PRODUCTGROUPID"].str.strip().str.upper()
        json_df["PRODUCTNAME"] = json_df["PRODUCTNAME"].str.strip()
        json_df["DELIVERYMONTH"] = json_df["DELIVERYMONTH"].str.strip()
        json_df = json_df.reindex(columns=["PRODUCTID", "DELIVERYMONTH", "PRESETTLEMENTPRICE", "OPENPRICE",
                                           "HIGHESTPRICE", "LOWESTPRICE", "CLOSEPRICE", "SETTLEMENTPRICE",
                                           "VOLUME", "OPENINTEREST", "OPENINTERESTCHG"])
        # 修改列头，返回
        json_df.columns = ["variety_en", "contract", "pre_settlement", "open_price", "highest", "lowest",
                           "close_price", "settlement", "trade_volume", "position_volume", "increase_volume"]
        # 合约等于品种+合约
        json_df["contract"] = json_df["variety_en"] + json_df["contract"]
        # 处理没有数据或空值=补0
        json_df.replace(to_replace="^\s*$", value=np.nan, regex=True, inplace=True)
        json_df = json_df.fillna(0)
        # 加入成交额字段
        json_df['trade_price'] = [0 for _ in range(json_df.shape[0])]
        daily_data = json_df.to_dict(orient='records')
        return daily_data, 'Parser Successfully!'


class DCEDailyQuotes(object):
    DCE_VARIETY_EN = {
        '豆一': 'A', '豆二': 'B', '胶合板': 'BB', '玉米': 'C', '玉米淀粉': 'CS', '苯乙烯': 'EB', '乙二醇': 'EG',
        '纤维板': 'FB', '铁矿石': 'I', '焦炭': 'J', '鸡蛋': 'JD', '焦煤': 'JM', '聚乙烯': 'L', '生猪': 'LH',
        '豆粕': 'M', '棕榈油': 'P', '液化石油气': 'PG', '聚丙烯': 'PP', '粳米': 'RR', '聚氯乙烯': 'V', '豆油': 'Y'
    }

    def __init__(self, parse_date: datetime.datetime):
        self.temp_folder = 'temp/dce/daily'
        self.parse_year = parse_date.strftime("%Y")
        self.parse_day = parse_date.strftime("%Y%m%d")
        self.url = "http://www.dce.com.cn/publicweb/quotesdata/exportDayQuotesChData.html"
        self.form_params = {
            "dayQuotes.variety": "all",
            "dayQuotes.trade_type": "0",
            "year": str(self.parse_year),
            "month": str(parse_date.month - 1),
            "day": parse_date.strftime("%d"),
            "exportFlag": "excel"
        }
        self.headers = {
            'User-Agent': 'User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
            'Host': 'www.dce.com.cn',
        }
        self.filepath = f'{self.temp_folder}/{self.parse_day}.xls'
        if not os.path.exists(self.temp_folder):
            os.makedirs(self.temp_folder)

    def get_quotes_xls_file(self):
        try:
            r = requests.post(url=self.url,data=self.form_params, headers=self.headers)
            r.encoding = 'utf8'
            content = r.content
            if r.status_code == 200:
                with open(self.filepath, 'wb') as f:
                    f.write(content)
            else:
                if os.path.exists(self.filepath):
                    os.remove(self.filepath)
        except Exception as e:
            logger.error(f'Spider {self.parse_day} DCE daily-quotes Error:{e}')
        else:
            logger.info(f'Spider {self.parse_day} DCE daily-quotes Successfully!')

    def get_daily_quotes(self):
        if not os.path.exists(self.filepath):
            self.get_quotes_xls_file()
        xls_df = pd.read_excel(self.filepath, thousands=',')
        # 判断表头格式
        if xls_df.columns.tolist() != ['商品名称', '交割月份', '开盘价', '最高价', '最低价', '收盘价', '前结算价', '结算价',
                                       '涨跌', '涨跌1', '成交量', '持仓量', '持仓量变化', '成交额']:
            return [], f'Get {self.parse_day} DCE daily-quotes Error: columns error!'
        # 选取无合计,小计,总计的行
        xls_df = xls_df[~xls_df["商品名称"].str.contains("小计|总计|合计")]
        # 交个月份转为int再转为str
        xls_df["交割月份"] = xls_df["交割月份"].apply(lambda x: str(int(x)))
        # 修改商品名称为英文
        xls_df["商品名称"] = xls_df["商品名称"].apply(lambda x: self.DCE_VARIETY_EN.get(x, '未知'))

        # 重置列头并重命名
        xls_df = xls_df.reindex(columns=["商品名称", "交割月份", "前结算价", "开盘价", "最高价", "最低价", "收盘价",
                                         "结算价", "成交量", "持仓量", "持仓量变化", "成交额"])
        xls_df.columns = ["variety_en", "contract", "pre_settlement", "open_price", "highest", "lowest",
                          "close_price", "settlement", "trade_volume", "position_volume", "increase_volume",
                          "trade_price"]
        # 合约改为品种+交割月的形式
        xls_df["contract"] = xls_df["variety_en"] + xls_df["contract"]
        daily_data = xls_df.to_dict(orient='records')
        return daily_data, f'Get {self.parse_day} DCE daily-quotes Successfully!'


class CFFEXDailyQuotes(object):
    def __init__(self, parse_date: datetime.datetime):
        self.temp_folder = 'temp/cffex/daily'
        self.parse_year = parse_date.strftime("%Y")
        self.parse_day = parse_date.strftime("%Y%m%d")
        self.url = f"http://www.cffex.com.cn/sj/hqsj/rtj/{self.parse_day[:-2]}/{self.parse_day[-2:]}/{self.parse_day}_1.csv"
        self.headers = {
            'User-Agent': 'User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
            'Referer': 'http://www.cffex.com.cn/rtj/',
        }
        self.filepath = f'{self.temp_folder}/{self.parse_day}.csv'
        if not os.path.exists(self.temp_folder):
            os.makedirs(self.temp_folder)

    def get_quotes_csv_file(self):
        try:
            r = requests.get(url=self.url, headers=self.headers)
            r.encoding = 'utf8'
            if r.status_code == 200:
                content = r.content
                with open(self.filepath, 'wb') as f:
                    f.write(content)
            else:
                if os.path.exists(self.filepath):
                    os.remove(self.filepath)
        except Exception as e:
            logger.error(f'Spider {self.parse_day} CFFEX daily-quotes Error:{e}')
        else:
            logger.info(f'Spider {self.parse_day} CFFEX daily-quotes Successfully!')

    def get_daily_quotes(self):
        if not os.path.exists(self.filepath):
            self.get_quotes_csv_file()
        csv_df = pd.read_csv(self.filepath, encoding='gbk', error_bad_lines=False)
        if csv_df.columns.tolist() != ['合约代码', '今开盘', '最高价', '最低价', '成交量',
                                              '成交金额', '持仓量', '持仓变化', '今收盘', '今结算', '前结算', '涨跌1', '涨跌2', 'Delta']:
            return [], f'Get {self.parse_day} CFFEX daily-quotes Error: columns error!'
        csv_df = csv_df[~csv_df['合约代码'].str.contains('-C-|-P-|小计|合计')]  # 去除合约代码-C- -P-的期权数据 小计 合计总计数据
        csv_df["合约代码"] = csv_df["合约代码"].str.strip()  # 去除前后空格
        csv_df["成交金额"] = csv_df["成交金额"].round(decimals=6)  # 成交金额保留6位小数
        csv_df["品种"] = csv_df["合约代码"].apply(hu_tool.split_number_en).apply(lambda x: x[0].upper())  # 使用合约代码列添加品种列
        # 重置索引
        csv_df = csv_df.reindex(columns=["品种", "合约代码", "前结算", "今开盘", "最高价", "最低价", "今收盘",
                                         "今结算", "涨跌1", "涨跌2", "成交量", "成交金额", "持仓量", "持仓变化"])
        csv_df.columns = ["variety_en", "contract", "pre_settlement", "open_price", "highest", "lowest",
                          "close_price", "settlement", "zd_1", "zd_2", "trade_volume", "trade_price",
                          "position_volume", "increase_volume"]
        # 填充空值
        csv_df = csv_df.fillna(0)
        daily_data = csv_df.to_dict(orient='records')
        return daily_data, f'Get {self.parse_day} CFFEX daily-quotes Successfully!'


if __name__ == '__main__':
    date = datetime.datetime.today() + datetime.timedelta(days=-1)
    # czce = CZCEDailyQuotes(parse_date=date)
    # data, msg = czce.get_daily_quotes()  # 郑商所

    # shfe = SHFEDailyQuotes(parse_date=date)  # 上期所
    # shfe.get_daily_quotes()

    # dce = DCEDailyQuotes(parse_date=date)  # 大商所
    # dce.get_daily_quotes()

    cffex = CFFEXDailyQuotes(parse_date=date)  # 中金所
    cffex.get_daily_quotes()



